{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import requests\n",
    "import time\n",
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "376\n",
      "360\n",
      "350\n",
      "670\n",
      "670\n",
      "560\n",
      "635\n",
      "370\n",
      "315\n",
      "337\n"
     ]
    }
   ],
   "source": [
    "#设置列表页URL的固定部分\n",
    "url='https://sz.lianjia.com/ershoufang/longhuaqu/'\n",
    "#设置页面页的可变部分\n",
    "page=('pg')\n",
    "#设置请求头部信息\n",
    "headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',\n",
    "'Accept':'text/html;q=0.9,*/*;q=0.8',\n",
    "'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',\n",
    "'Accept-Encoding':'gzip',\n",
    "'Connection':'close',\n",
    "'Referer':'http://www.baidu.com/link?url=_andhfsjjjKRgEWkj7i9cFmYYGsisrnm2A-TN3XZDQXxvGsM9k9ZZSnikW2Yds4s&amp;amp;wd=&amp;amp;eqid=c3435a7d00146bd600000003582bfd1f'\n",
    "}\n",
    "for i in range(1,40):\n",
    "    if i == 1:\n",
    "        i=str(i)\n",
    "        a=(url+page+i+'/')\n",
    "        r=requests.get(url=a,headers=headers)\n",
    "        html=r.content\n",
    "    else:\n",
    "        i=str(i)\n",
    "        a=(url+page+i+'/')\n",
    "        r=requests.get(url=a,headers=headers)\n",
    "        html2=r.content\n",
    "        html = html + html2\n",
    "    #每次间隔0.5秒\n",
    "    time.sleep(0.5)\n",
    "encoding=requests.get(url,headers=headers).encoding\n",
    "lj=etree.HTML(html,parser=etree.HTMLParser(encoding=encoding))\n",
    "#提取房源总价\n",
    "price=lj.xpath('//div[@class=\"totalPrice\"]')\n",
    "\n",
    "tp=[]\n",
    "for a in price:\n",
    "    totalPrice=a.xpath('.//span/text()')[0]\n",
    "    tp.append(totalPrice)\n",
    "for p in tp[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "单价55246元/平米\n",
      "单价52848元/平米\n",
      "单价43783元/平米\n",
      "单价75248元/平米\n",
      "单价75256元/平米\n",
      "单价40984元/平米\n",
      "单价70745元/平米\n",
      "单价42228元/平米\n",
      "单价42209元/平米\n",
      "单价44964元/平米\n"
     ]
    }
   ],
   "source": [
    "#提取房源单价\n",
    "price=lj.xpath('//div[@class=\"unitPrice\"]')\n",
    "\n",
    "up=[]\n",
    "for a in price:\n",
    "    unitPrice=a.xpath('.//span/text()')[0]\n",
    "    up.append(unitPrice)\n",
    "for p in up[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "潜龙鑫茂花园B区 \n",
      "保利悦都 \n",
      "劲力城市明珠 \n",
      "万科金域华府 \n",
      "万科金域华府 \n",
      "锦绣江南三期 \n",
      "绿景公馆1866南区 \n",
      "金地鹭湖1号 \n",
      "风和日丽 \n",
      "美丽家园 \n"
     ]
    }
   ],
   "source": [
    "#提取房源位置\n",
    "weizhi=lj.xpath('//div[@class=\"houseInfo\"]')\n",
    "\n",
    "xiaoqu=[]\n",
    "for a in weizhi:\n",
    "    ar=a.xpath('.//a/text()')[0]\n",
    "    xiaoqu.append(ar)\n",
    "for p in xiaoqu[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " | 2室2厅 | 68.06平米 | 北 | 精装 | 有电梯\n",
      " | 2室1厅 | 68.12平米 | 西南 | 精装\n",
      " | 3室1厅 | 79.94平米 | 东南 | 精装 | 有电梯\n",
      " | 3室2厅 | 89.04平米 | 西南 | 精装 | 有电梯\n",
      " | 3室2厅 | 89.03平米 | 西南 | 精装 | 有电梯\n",
      " | 4室1厅 | 136.64平米 | 东南 | 简装 | 有电梯\n",
      " | 3室2厅 | 89.76平米 | 南 | 其他 | 有电梯\n",
      " | 4室2厅 | 87.62平米 | 西南 | 精装 | 有电梯\n",
      " | 2室2厅 | 74.63平米 | 东 | 简装 | 无电梯\n",
      " | 2室1厅 | 74.95平米 | 东南 | 简装 | 有电梯\n"
     ]
    }
   ],
   "source": [
    "#提取房源详情\n",
    "houseinfo=lj.xpath('//div[@class=\"houseInfo\"]')\n",
    "\n",
    "hi=[]\n",
    "for a in houseinfo:\n",
    "    info=a.xpath('./text()')[0]\n",
    "    hi.append(info)\n",
    "for p in hi[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "民治\n",
      "龙华中心\n",
      "龙华中心\n",
      "深圳北站\n",
      "深圳北站\n",
      "深圳北站\n",
      "深圳北站\n",
      "观澜\n",
      "深圳北站\n",
      "龙华中心\n"
     ]
    }
   ],
   "source": [
    "#提取房源区域\n",
    "quyu=lj.xpath('//div[@class=\"positionInfo\"]')\n",
    "\n",
    "area=[]\n",
    "for a in quyu:\n",
    "    qu=a.xpath('.//a/text()')[0]\n",
    "    area.append(qu)\n",
    "for p in area[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "高楼层(共16层)2006年建板塔结合\n",
      "中楼层(共27层)2016年建板塔结合\n",
      "中楼层(共12层)2005年建板塔结合\n",
      "高楼层(共32层)2010年建板塔结合\n",
      "中楼层(共32层)2010年建板塔结合\n",
      "低楼层(共15层)2005年建板楼\n",
      "高楼层(共28层)2013年建板塔结合\n",
      "高楼层(共16层)2016年建板塔结合\n",
      "中楼层(共8层)2002年建塔楼\n",
      "高楼层(共12层)2002年建板塔结合\n"
     ]
    }
   ],
   "source": [
    "#提取房源楼况\n",
    "building=lj.xpath('//div[@class=\"positionInfo\"]')\n",
    "\n",
    "loukuang=[]\n",
    "for a in building:\n",
    "    bu=a.xpath('./text()')[0]\n",
    "    loukuang.append(bu.split(' ')[0])\n",
    "for p in loukuang[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66人关注 / 共20次带看 / 5个月以前发布\n",
      "190人关注 / 共18次带看 / 8个月以前发布\n",
      "60人关注 / 共31次带看 / 3个月以前发布\n",
      "365人关注 / 共16次带看 / 7个月以前发布\n",
      "276人关注 / 共2次带看 / 8个月以前发布\n",
      "12人关注 / 共0次带看 / 24天以前发布\n",
      "90人关注 / 共12次带看 / 一年前发布\n",
      "189人关注 / 共22次带看 / 6个月以前发布\n",
      "289人关注 / 共11次带看 / 4个月以前发布\n",
      "93人关注 / 共16次带看 / 4个月以前发布\n"
     ]
    }
   ],
   "source": [
    "#提取房源关注\n",
    "guanzhu=lj.xpath('//div[@class=\"followInfo\"]')\n",
    "\n",
    "follow=[]\n",
    "for a in guanzhu:\n",
    "    gu=a.xpath('./text()')[0]\n",
    "    follow.append(gu)\n",
    "for p in follow[:10]:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['55246',\n",
       " '52848',\n",
       " '43783',\n",
       " '75248',\n",
       " '75256',\n",
       " '40984',\n",
       " '70745',\n",
       " '42228',\n",
       " '42209',\n",
       " '44964']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#处理房源单价\n",
    "import re\n",
    "re.split('[\"单价\",\"元/平米\"]',up[20])[2]\n",
    "unitprice = [re.split('[\"单价\",\"元/平米\"]',x)[2] for x in up]\n",
    "unitprice[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#处理房源详情\n",
    "rooms=[]\n",
    "size=[]\n",
    "chaoxiang=[]\n",
    "zhuangxiu=[]\n",
    "dianti=[]\n",
    "for x in hi:\n",
    "    y = x.split(' | ')\n",
    "    if len(y)==6:\n",
    "        rooms.append(y[1])\n",
    "        size.append(y[2])\n",
    "        chaoxiang.append(y[3])\n",
    "        zhuangxiu.append(y[4])\n",
    "        dianti.append(y[5])\n",
    "    else:\n",
    "        rooms.append(y[1])\n",
    "        size.append(y[2])\n",
    "        chaoxiang.append(y[3])\n",
    "        zhuangxiu.append(y[4])\n",
    "        dianti.append(None)\n",
    "size = [x.split('平米')[0] for x in size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#处理房源楼况\n",
    "loukuang[0]\n",
    "height=[]\n",
    "year=[]\n",
    "build_type=[]\n",
    "for x in loukuang:\n",
    "    height.append(x.split('楼')[0])\n",
    "    if '塔楼' in x:\n",
    "        build_type.append('塔楼')\n",
    "    elif '板楼' in x:\n",
    "        build_type.append('板楼')\n",
    "    elif '板塔结合' in x:\n",
    "        build_type.append('板塔结合')\n",
    "    else:\n",
    "        build_type.append(None)\n",
    "    if re.search(r'\\d{4}',x) is None:\n",
    "        year.append(None)\n",
    "    else:\n",
    "        year.append(re.search(r'\\d{4}',x).group())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['66', '190', '60', '365', '276', '12', '90', '189', '289', '93']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#处理房源关注\n",
    "follow = [x.split('人关注')[0] for x in follow]\n",
    "follow[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>totalprice</th>\n",
       "      <th>unitprice</th>\n",
       "      <th>position</th>\n",
       "      <th>rooms</th>\n",
       "      <th>size</th>\n",
       "      <th>orientation</th>\n",
       "      <th>decoration</th>\n",
       "      <th>elevator</th>\n",
       "      <th>area</th>\n",
       "      <th>height</th>\n",
       "      <th>year</th>\n",
       "      <th>buildtype</th>\n",
       "      <th>follow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>376</td>\n",
       "      <td>55246</td>\n",
       "      <td>潜龙鑫茂花园B区</td>\n",
       "      <td>2室2厅</td>\n",
       "      <td>68.06</td>\n",
       "      <td>北</td>\n",
       "      <td>精装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>民治</td>\n",
       "      <td>高</td>\n",
       "      <td>2006</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>360</td>\n",
       "      <td>52848</td>\n",
       "      <td>保利悦都</td>\n",
       "      <td>2室1厅</td>\n",
       "      <td>68.12</td>\n",
       "      <td>西南</td>\n",
       "      <td>精装</td>\n",
       "      <td>None</td>\n",
       "      <td>龙华中心</td>\n",
       "      <td>中</td>\n",
       "      <td>2016</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>350</td>\n",
       "      <td>43783</td>\n",
       "      <td>劲力城市明珠</td>\n",
       "      <td>3室1厅</td>\n",
       "      <td>79.94</td>\n",
       "      <td>东南</td>\n",
       "      <td>精装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>龙华中心</td>\n",
       "      <td>中</td>\n",
       "      <td>2005</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>670</td>\n",
       "      <td>75248</td>\n",
       "      <td>万科金域华府</td>\n",
       "      <td>3室2厅</td>\n",
       "      <td>89.04</td>\n",
       "      <td>西南</td>\n",
       "      <td>精装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>深圳北站</td>\n",
       "      <td>高</td>\n",
       "      <td>2010</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>670</td>\n",
       "      <td>75256</td>\n",
       "      <td>万科金域华府</td>\n",
       "      <td>3室2厅</td>\n",
       "      <td>89.03</td>\n",
       "      <td>西南</td>\n",
       "      <td>精装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>深圳北站</td>\n",
       "      <td>中</td>\n",
       "      <td>2010</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>560</td>\n",
       "      <td>40984</td>\n",
       "      <td>锦绣江南三期</td>\n",
       "      <td>4室1厅</td>\n",
       "      <td>136.64</td>\n",
       "      <td>东南</td>\n",
       "      <td>简装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>深圳北站</td>\n",
       "      <td>低</td>\n",
       "      <td>2005</td>\n",
       "      <td>板楼</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>635</td>\n",
       "      <td>70745</td>\n",
       "      <td>绿景公馆1866南区</td>\n",
       "      <td>3室2厅</td>\n",
       "      <td>89.76</td>\n",
       "      <td>南</td>\n",
       "      <td>其他</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>深圳北站</td>\n",
       "      <td>高</td>\n",
       "      <td>2013</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>370</td>\n",
       "      <td>42228</td>\n",
       "      <td>金地鹭湖1号</td>\n",
       "      <td>4室2厅</td>\n",
       "      <td>87.62</td>\n",
       "      <td>西南</td>\n",
       "      <td>精装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>观澜</td>\n",
       "      <td>高</td>\n",
       "      <td>2016</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>315</td>\n",
       "      <td>42209</td>\n",
       "      <td>风和日丽</td>\n",
       "      <td>2室2厅</td>\n",
       "      <td>74.63</td>\n",
       "      <td>东</td>\n",
       "      <td>简装</td>\n",
       "      <td>无电梯</td>\n",
       "      <td>深圳北站</td>\n",
       "      <td>中</td>\n",
       "      <td>2002</td>\n",
       "      <td>塔楼</td>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>337</td>\n",
       "      <td>44964</td>\n",
       "      <td>美丽家园</td>\n",
       "      <td>2室1厅</td>\n",
       "      <td>74.95</td>\n",
       "      <td>东南</td>\n",
       "      <td>简装</td>\n",
       "      <td>有电梯</td>\n",
       "      <td>龙华中心</td>\n",
       "      <td>高</td>\n",
       "      <td>2002</td>\n",
       "      <td>板塔结合</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  totalprice unitprice     position rooms    size orientation decoration  \\\n",
       "0        376     55246    潜龙鑫茂花园B区   2室2厅   68.06           北         精装   \n",
       "1        360     52848        保利悦都   2室1厅   68.12          西南         精装   \n",
       "2        350     43783      劲力城市明珠   3室1厅   79.94          东南         精装   \n",
       "3        670     75248      万科金域华府   3室2厅   89.04          西南         精装   \n",
       "4        670     75256      万科金域华府   3室2厅   89.03          西南         精装   \n",
       "5        560     40984      锦绣江南三期   4室1厅  136.64          东南         简装   \n",
       "6        635     70745  绿景公馆1866南区   3室2厅   89.76           南         其他   \n",
       "7        370     42228      金地鹭湖1号   4室2厅   87.62          西南         精装   \n",
       "8        315     42209        风和日丽   2室2厅   74.63           东         简装   \n",
       "9        337     44964        美丽家园   2室1厅   74.95          东南         简装   \n",
       "\n",
       "  elevator  area height  year buildtype follow  \n",
       "0      有电梯    民治      高  2006      板塔结合     66  \n",
       "1     None  龙华中心      中  2016      板塔结合    190  \n",
       "2      有电梯  龙华中心      中  2005      板塔结合     60  \n",
       "3      有电梯  深圳北站      高  2010      板塔结合    365  \n",
       "4      有电梯  深圳北站      中  2010      板塔结合    276  \n",
       "5      有电梯  深圳北站      低  2005        板楼     12  \n",
       "6      有电梯  深圳北站      高  2013      板塔结合     90  \n",
       "7      有电梯    观澜      高  2016      板塔结合    189  \n",
       "8      无电梯  深圳北站      中  2002        塔楼    289  \n",
       "9      有电梯  龙华中心      高  2002      板塔结合     93  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "house=pd.DataFrame({'totalprice':tp,'unitprice':unitprice,'position':xiaoqu,'rooms':rooms,'size':size,'orientation':chaoxiang,'decoration':zhuangxiu,'elevator':dianti,'area':area,'height':height,'year':year,'buildtype':build_type,'follow':follow})\n",
    "house[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    1004\n",
       "Name: size, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 缺失值处理(全部删掉)\n",
    "house.columns.tolist()\n",
    "for x in house.columns.tolist():\n",
    "    house[x]=house[x].fillna('null')\n",
    "    a = house[house[x]=='null'].index.tolist()\n",
    "    house = house.drop(a)\n",
    "house['size'].isnull().value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理数据类型\n",
    "house['totalprice']=pd.to_numeric(house['totalprice'],errors='coerce')\n",
    "house['unitprice']=pd.to_numeric(house['unitprice'],errors='coerce')\n",
    "house['size']=pd.to_numeric(house['size'],errors='coerce')\n",
    "house['year']=pd.to_numeric(house['year'],errors='coerce')\n",
    "house['follow']=pd.to_numeric(house['follow'],errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "house.to_excel('house_longhua.xls',sheet_name='biubiu')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
